This notebook is all about benchmarking some R code used in this package.
Hardware / Software used:
- Intel i7-4600U
- Compilation flags for C/C++:
-O2 -Wall $(DEBUGFLAG) -mtune=core2 (R’s defaults)
- Windows Server 2012 R2
- R 3.3.2 + Intel MKL
Libraries
library(data.table)
data.table 1.10.4
The fastest way to learn (by data.table authors): https://www.datacamp.com/courses/data-analysis-the-data-table-way
Documentation: ?data.table, example(data.table) and browseVignettes("data.table")
Release notes, videos and slides: http://r-datatable.com
library(microbenchmark)
library(Rcpp)
library(ggplot2)
library(plotly)
Attaching package: <U+393C><U+3E31>plotly<U+393C><U+3E32>
The following object is masked from <U+393C><U+3E31>package:ggplot2<U+393C><U+3E32>:
last_plot
The following object is masked from <U+393C><U+3E31>package:stats<U+393C><U+3E32>:
filter
The following object is masked from <U+393C><U+3E31>package:graphics<U+393C><U+3E32>:
layout
# Helper function to print data well in tables
print_well <- function(data, digits = 6) {
# To milliseconds
data <- data / 1000000
# Sprintf helper
sprintf_helper <- paste0("%.0", digits, "f")
cat("| Min | 25% | 50% | 75% | Max | Mean | \n| --: | --: | --: | --: | --: | --: | \n| ", sprintf(sprintf_helper, min(data)), " | ", sprintf(sprintf_helper, quantile(data, probs = 0.25)), " | ", sprintf(sprintf_helper, median(data)), " | ", sprintf(sprintf_helper, quantile(data, probs = 0.75)), " | ", sprintf(sprintf_helper, max(data)), " | ", sprintf(sprintf_helper, mean(data)), " | \n", sep = "")
return(data)
}
# Test case function
# Arguments renamed to avoid recursive clash
test_case <- function(f, preds, labels, eps) {
cat("Test case: ", do.call(f, list(preds = preds[1:50],
labels = labels[1:50],
eps = 1e-15)), " \n")
}
Benchmarking Clamped Vector to Logloss
For a 2-class vector of 1,000,000 observations:
- Vector A of length=(1000000)
- Vector B of length=(1000000) with 2 classes
A = [1, 2, 3, 4, ..., 1000000]
B = [0, 1, 1, 0, ...]
Get the following Vector C and D:
C = Clamped A by 1e-15
D = Mean of logloss(C, B)
Initialize data
# Generate random data
set.seed(11111)
data <- runif(1000000, 0, 1)
labels <- round(runif(1000000, 0, 1), digits = 0)
# How many digits for benchmarking in milliseconds
my_digits <- 6L
# How many runs for benchmarking?
my_runs <- 1000L
Benchmarks
# ===== BLOCK 1 =====
faster1 <- function(preds, labels, eps = 1e-15) {
x <- preds
x[x < eps] <- eps
x[x > (1 - eps)] <- 1 - eps
return(-mean(labels * log(x) + (1 - labels) * log(1 - x)))
}
test_case(faster1, preds = data, labels = labels, eps = 1e-15)
Test case: 0.9837966
data1 <- print_well(microbenchmark(faster1(data, labels), times = my_runs)$time, digits = my_digits)
| 98.236812 |
102.558966 |
104.827241 |
109.353338 |
261.528689 |
110.746225 |
# ===== BLOCK 2 =====
faster2 <- function(preds, labels, eps = 1e-15) {
x <- pmin(pmax(preds, eps), 1 - eps)
return(-mean(labels * log(x) + (1 - labels) * log(1 - x)))
}
test_case(faster2, preds = data, labels = labels, eps = 1e-15)
Test case: 0.9837966
data2 <- print_well(microbenchmark(faster2(data, labels), times = my_runs)$time, digits = my_digits)
| 100.001406 |
104.717001 |
108.238398 |
117.771084 |
210.083657 |
121.705744 |
# ===== BLOCK 3 =====
faster3 <- function(preds, labels, eps = 1e-15) {
x <- preds
x[x < eps] <- eps
x[x > (1 - eps)] <- 1 - eps
return(-1/length(labels) * (sum(labels * log(x) + (1 - labels) * log(1 - x))))
}
test_case(faster3, preds = data, labels = labels, eps = 1e-15)
Test case: 0.9837966
data3 <- print_well(microbenchmark(faster3(data, labels), times = my_runs)$time, digits = my_digits)
| 96.428882 |
101.448587 |
103.667824 |
107.993494 |
199.984946 |
109.273259 |
# ===== BLOCK 4 =====
faster4 <- function(preds, labels, eps = 1e-15) {
x <- pmin(pmax(preds, eps), 1 - eps)
return(-1/length(labels) * (sum(labels * log(x) + (1 - labels) * log(1 - x))))
}
test_case(faster4, preds = data, labels = labels, eps = 1e-15)
Test case: 0.9837966
data4 <- print_well(microbenchmark(faster4(data, labels), times = my_runs)$time, digits = my_digits)
| 98.875441 |
103.396217 |
106.065157 |
111.215343 |
191.767533 |
114.586109 |
# ===== BLOCK 5 =====
cppFunction("double faster5(NumericVector preds, NumericVector labels, double eps) {
NumericVector clamped = clamp(eps, preds, 1 - eps);
NumericVector loggy = -1 * ((labels * log(clamped) + (1 - labels) * log(1 - clamped)));
double logloss = mean(loggy);
return logloss;
}")
test_case(faster5, preds = data, labels = labels, eps = 1e-15)
Test case: 0.9837966
data5 <- print_well(microbenchmark(faster5(data, labels, eps = 1e-15), times = my_runs)$time, digits = my_digits)
| 78.746066 |
80.047844 |
81.832206 |
87.750363 |
166.269487 |
84.050458 |
# ===== BLOCK 6 =====
cppFunction("double faster6(NumericVector preds, NumericVector labels, double eps) {
NumericVector clamped = clamp(eps, preds, 1 - eps);
NumericVector loggy = -1 * ((labels * log(clamped) + (1 - labels) * log(1 - clamped)));
double logloss = sum(loggy)/loggy.size();
return logloss;
}")
test_case(faster6, preds = data, labels = labels, eps = 1e-15)
Test case: 0.9837966
data6 <- print_well(microbenchmark(faster6(data, labels, eps = 1e-15), times = my_runs)$time, digits = my_digits)
| 77.733382 |
79.289472 |
81.178370 |
87.284316 |
165.628577 |
83.651944 |
Summary Results
data_time <- data.table(rbindlist(list(data.frame(Time = data1, Bench = "faster1"),
data.frame(Time = data2, Bench = "faster2"),
data.frame(Time = data3, Bench = "faster3"),
data.frame(Time = data4, Bench = "faster4"),
data.frame(Time = data5, Bench = "faster5"),
data.frame(Time = data6, Bench = "faster6"))))
data_time <- data_time[, t_mean := mean(Time), by = Bench]
data_time <- data_time[, t_median := median(Time), by = Bench]
data_time$Benchs <- data_time$Bench
levels(data_time$Benchs) <- paste0("faster", 1:6, "= [", sprintf(paste0("%.0", my_digits, "f"), data_time[, list(min(Time)), by = Bench]$V1), ", ", sprintf(paste0("%.0", my_digits, "f"), data_time[, list(max(Time)), by = Bench]$V1), "], mean=", sprintf(paste0("%.0", my_digits, "f"), data_time[, list(mean(Time)), by = Bench]$V1), ", median=", sprintf(paste0("%.0", my_digits, "f"), data_time[, list(median(Time)), by = Bench]$V1))
my_time <- data_time[, list(min(Time), quantile(Time, probs = 0.25), median(Time), quantile(Time, probs = 0.75), max(Time), mean(Time)), by = Bench]
colnames(my_time) <- c("Function", "Min", "25%", "50%", "75%", "Max", "Mean")
my_time <- my_time[order(Mean, decreasing = FALSE), ]
print(my_time, digits = 6)
Plot Results
ggplotly(ggplot(data = data_time, aes(x = Time)) + geom_histogram(aes(y = ..density..), bins = 20, color = "darkblue", fill = "lightblue") + facet_wrap(~ Benchs, ncol = 2) + geom_vline(aes(xintercept = t_mean), color = "blue", linetype = "dashed", size = 2) + geom_vline(aes(xintercept = t_median), color = "red", linetype = "dashed", size = 2) + labs(x = "Time (Milliseconds)", y = "Density") + theme_bw())
ggplotly(ggplot(data = data_time[, .(Time, Bench)], aes(x = Time, y = ..count.., fill = Bench)) + geom_histogram(aes(y = ..density..), bins = 100, position = "fill") + labs(x = "Time (Milliseconds)", y = "Density") + theme_bw())
ggplotly(ggplot(data = data_time[, .(Time, Bench)], aes(x = Time, y = ..count.., fill = Bench)) + geom_density(position = "fill") + labs(x = "Time (Milliseconds)", y = "Density") + theme_bw())
LS0tDQp0aXRsZTogIkJlbmNobWFya3M6IExvZ2xvc3MiDQpvdXRwdXQ6DQogIGh0bWxfbm90ZWJvb2s6DQogICAgY29sbGFwc2VkOiBubw0KICAgIHRoZW1lOiB1bml0ZWQNCiAgICB0b2M6IHllcw0KICAgIHRvY19kZXB0aDogNg0KICAgIHRvY19mbG9hdDogeWVzDQoNCi0tLQ0KDQpUaGlzIG5vdGVib29rIGlzIGFsbCBhYm91dCBiZW5jaG1hcmtpbmcgc29tZSBSIGNvZGUgdXNlZCBpbiB0aGlzIHBhY2thZ2UuDQoNCkhhcmR3YXJlIC8gU29mdHdhcmUgdXNlZDoNCg0KKiBJbnRlbCBpNy00NjAwVQ0KKiBDb21waWxhdGlvbiBmbGFncyBmb3IgQy9DKys6IGAtTzIgLVdhbGwgJChERUJVR0ZMQUcpIC1tdHVuZT1jb3JlMmAgKFIncyBkZWZhdWx0cykNCiogV2luZG93cyBTZXJ2ZXIgMjAxMiBSMg0KKiBSIDMuMy4yICsgSW50ZWwgTUtMDQoNCiMgTGlicmFyaWVzDQoNCmBgYHtyIGluaXR9DQpsaWJyYXJ5KGRhdGEudGFibGUpDQpsaWJyYXJ5KG1pY3JvYmVuY2htYXJrKQ0KbGlicmFyeShSY3BwKQ0KbGlicmFyeShnZ3Bsb3QyKQ0KbGlicmFyeShwbG90bHkpDQpgYGANCg0KYGBge3IgYmFzZWR9DQoNCiMgSGVscGVyIGZ1bmN0aW9uIHRvIHByaW50IGRhdGEgd2VsbCBpbiB0YWJsZXMNCnByaW50X3dlbGwgPC0gZnVuY3Rpb24oZGF0YSwgZGlnaXRzID0gNikgew0KICANCiAgIyBUbyBtaWxsaXNlY29uZHMNCiAgZGF0YSA8LSBkYXRhIC8gMTAwMDAwMA0KICANCiAgIyBTcHJpbnRmIGhlbHBlcg0KICBzcHJpbnRmX2hlbHBlciA8LSBwYXN0ZTAoIiUuMCIsIGRpZ2l0cywgImYiKQ0KICANCiAgY2F0KCJ8IE1pbiB8IDI1JSB8IDUwJSB8IDc1JSB8IE1heCB8IE1lYW4gfCAgXG58IC0tOiB8IC0tOiB8IC0tOiB8IC0tOiB8IC0tOiB8IC0tOiB8ICBcbnwgIiwgc3ByaW50ZihzcHJpbnRmX2hlbHBlciwgbWluKGRhdGEpKSwgIiB8ICIsIHNwcmludGYoc3ByaW50Zl9oZWxwZXIsIHF1YW50aWxlKGRhdGEsIHByb2JzID0gMC4yNSkpLCAiIHwgIiwgc3ByaW50ZihzcHJpbnRmX2hlbHBlciwgbWVkaWFuKGRhdGEpKSwgIiB8ICIsIHNwcmludGYoc3ByaW50Zl9oZWxwZXIsIHF1YW50aWxlKGRhdGEsIHByb2JzID0gMC43NSkpLCAiIHwgIiwgc3ByaW50ZihzcHJpbnRmX2hlbHBlciwgbWF4KGRhdGEpKSwgIiB8ICIsIHNwcmludGYoc3ByaW50Zl9oZWxwZXIsIG1lYW4oZGF0YSkpLCAiIHwgIFxuIiwgc2VwID0gIiIpDQogIA0KICByZXR1cm4oZGF0YSkNCiAgDQp9DQoNCiMgVGVzdCBjYXNlIGZ1bmN0aW9uDQojIEFyZ3VtZW50cyByZW5hbWVkIHRvIGF2b2lkIHJlY3Vyc2l2ZSBjbGFzaA0KdGVzdF9jYXNlIDwtIGZ1bmN0aW9uKGYsIHByZWRzLCBsYWJlbHMsIGVwcykgew0KICBjYXQoIlRlc3QgY2FzZTogIiwgZG8uY2FsbChmLCBsaXN0KHByZWRzID0gcHJlZHNbMTo1MF0sDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGFiZWxzID0gbGFiZWxzWzE6NTBdLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGVwcyA9IDFlLTE1KSksICIgIFxuIikNCn0NCg0KYGBgDQoNCiMgQmVuY2htYXJraW5nIENsYW1wZWQgVmVjdG9yIHRvIExvZ2xvc3MNCg0KRm9yIGEgMi1jbGFzcyB2ZWN0b3Igb2YgMSwwMDAsMDAwIG9ic2VydmF0aW9uczoNCg0KKiBWZWN0b3IgQSBvZiBsZW5ndGg9KDEwMDAwMDApDQoqIFZlY3RvciBCIG9mIGxlbmd0aD0oMTAwMDAwMCkgd2l0aCAyIGNsYXNzZXMNCg0KYGBgDQpBID0gWzEsIDIsIDMsIDQsIC4uLiwgMTAwMDAwMF0NCkIgPSBbMCwgMSwgMSwgMCwgLi4uXQ0KYGBgDQoNCkdldCB0aGUgZm9sbG93aW5nIFZlY3RvciBDIGFuZCBEOg0KDQpgYGANCkMgPSBDbGFtcGVkIEEgYnkgMWUtMTUNCkQgPSBNZWFuIG9mIGxvZ2xvc3MoQywgQikNCmBgYA0KDQojIEluaXRpYWxpemUgZGF0YQ0KDQpgYGB7ciBiZW5jaDF9DQojIEdlbmVyYXRlIHJhbmRvbSBkYXRhDQpzZXQuc2VlZCgxMTExMSkNCmRhdGEgPC0gcnVuaWYoMTAwMDAwMCwgMCwgMSkNCmxhYmVscyA8LSByb3VuZChydW5pZigxMDAwMDAwLCAwLCAxKSwgZGlnaXRzID0gMCkNCg0KIyBIb3cgbWFueSBkaWdpdHMgZm9yIGJlbmNobWFya2luZyBpbiBtaWxsaXNlY29uZHMNCm15X2RpZ2l0cyA8LSA2TA0KDQojIEhvdyBtYW55IHJ1bnMgZm9yIGJlbmNobWFya2luZz8NCm15X3J1bnMgPC0gMTAwMEwNCmBgYA0KDQojIEJlbmNobWFya3MNCg0KYGBge3IgYmVuY2gyLCByZXN1bHRzPSJhc2lzIn0NCg0KIyA9PT09PSBCTE9DSyAxID09PT09DQpmYXN0ZXIxIDwtIGZ1bmN0aW9uKHByZWRzLCBsYWJlbHMsIGVwcyA9IDFlLTE1KSB7DQogIHggPC0gcHJlZHMNCiAgeFt4IDwgZXBzXSA8LSBlcHMNCiAgeFt4ID4gKDEgLSBlcHMpXSA8LSAxIC0gZXBzDQogIHJldHVybigtbWVhbihsYWJlbHMgKiBsb2coeCkgKyAoMSAtIGxhYmVscykgKiBsb2coMSAtIHgpKSkNCn0NCnRlc3RfY2FzZShmYXN0ZXIxLCBwcmVkcyA9IGRhdGEsIGxhYmVscyA9IGxhYmVscywgZXBzID0gMWUtMTUpDQpkYXRhMSA8LSBwcmludF93ZWxsKG1pY3JvYmVuY2htYXJrKGZhc3RlcjEoZGF0YSwgbGFiZWxzKSwgdGltZXMgPSBteV9ydW5zKSR0aW1lLCBkaWdpdHMgPSBteV9kaWdpdHMpDQoNCiMgPT09PT0gQkxPQ0sgMiA9PT09PQ0KZmFzdGVyMiA8LSBmdW5jdGlvbihwcmVkcywgbGFiZWxzLCBlcHMgPSAxZS0xNSkgew0KICB4IDwtIHBtaW4ocG1heChwcmVkcywgZXBzKSwgMSAtIGVwcykNCiAgcmV0dXJuKC1tZWFuKGxhYmVscyAqIGxvZyh4KSArICgxIC0gbGFiZWxzKSAqIGxvZygxIC0geCkpKQ0KfQ0KdGVzdF9jYXNlKGZhc3RlcjIsIHByZWRzID0gZGF0YSwgbGFiZWxzID0gbGFiZWxzLCBlcHMgPSAxZS0xNSkNCmRhdGEyIDwtIHByaW50X3dlbGwobWljcm9iZW5jaG1hcmsoZmFzdGVyMihkYXRhLCBsYWJlbHMpLCB0aW1lcyA9IG15X3J1bnMpJHRpbWUsIGRpZ2l0cyA9IG15X2RpZ2l0cykNCg0KIyA9PT09PSBCTE9DSyAzID09PT09DQpmYXN0ZXIzIDwtIGZ1bmN0aW9uKHByZWRzLCBsYWJlbHMsIGVwcyA9IDFlLTE1KSB7DQogIHggPC0gcHJlZHMNCiAgeFt4IDwgZXBzXSA8LSBlcHMNCiAgeFt4ID4gKDEgLSBlcHMpXSA8LSAxIC0gZXBzDQogIHJldHVybigtMS9sZW5ndGgobGFiZWxzKSAqIChzdW0obGFiZWxzICogbG9nKHgpICsgKDEgLSBsYWJlbHMpICogbG9nKDEgLSB4KSkpKQ0KfQ0KdGVzdF9jYXNlKGZhc3RlcjMsIHByZWRzID0gZGF0YSwgbGFiZWxzID0gbGFiZWxzLCBlcHMgPSAxZS0xNSkNCmRhdGEzIDwtIHByaW50X3dlbGwobWljcm9iZW5jaG1hcmsoZmFzdGVyMyhkYXRhLCBsYWJlbHMpLCB0aW1lcyA9IG15X3J1bnMpJHRpbWUsIGRpZ2l0cyA9IG15X2RpZ2l0cykNCg0KIyA9PT09PSBCTE9DSyA0ID09PT09DQpmYXN0ZXI0IDwtIGZ1bmN0aW9uKHByZWRzLCBsYWJlbHMsIGVwcyA9IDFlLTE1KSB7DQogIHggPC0gcG1pbihwbWF4KHByZWRzLCBlcHMpLCAxIC0gZXBzKQ0KICByZXR1cm4oLTEvbGVuZ3RoKGxhYmVscykgKiAoc3VtKGxhYmVscyAqIGxvZyh4KSArICgxIC0gbGFiZWxzKSAqIGxvZygxIC0geCkpKSkNCn0NCnRlc3RfY2FzZShmYXN0ZXI0LCBwcmVkcyA9IGRhdGEsIGxhYmVscyA9IGxhYmVscywgZXBzID0gMWUtMTUpDQpkYXRhNCA8LSBwcmludF93ZWxsKG1pY3JvYmVuY2htYXJrKGZhc3RlcjQoZGF0YSwgbGFiZWxzKSwgdGltZXMgPSBteV9ydW5zKSR0aW1lLCBkaWdpdHMgPSBteV9kaWdpdHMpDQoNCiMgPT09PT0gQkxPQ0sgNSA9PT09PQ0KY3BwRnVuY3Rpb24oImRvdWJsZSBmYXN0ZXI1KE51bWVyaWNWZWN0b3IgcHJlZHMsIE51bWVyaWNWZWN0b3IgbGFiZWxzLCBkb3VibGUgZXBzKSB7DQogIE51bWVyaWNWZWN0b3IgY2xhbXBlZCA9IGNsYW1wKGVwcywgcHJlZHMsIDEgLSBlcHMpOw0KICBOdW1lcmljVmVjdG9yIGxvZ2d5ID0gLTEgKiAoKGxhYmVscyAqIGxvZyhjbGFtcGVkKSArICgxIC0gbGFiZWxzKSAqIGxvZygxIC0gY2xhbXBlZCkpKTsNCiAgZG91YmxlIGxvZ2xvc3MgPSBtZWFuKGxvZ2d5KTsNCiAgcmV0dXJuIGxvZ2xvc3M7DQp9IikNCnRlc3RfY2FzZShmYXN0ZXI1LCBwcmVkcyA9IGRhdGEsIGxhYmVscyA9IGxhYmVscywgZXBzID0gMWUtMTUpDQpkYXRhNSA8LSBwcmludF93ZWxsKG1pY3JvYmVuY2htYXJrKGZhc3RlcjUoZGF0YSwgbGFiZWxzLCBlcHMgPSAxZS0xNSksIHRpbWVzID0gbXlfcnVucykkdGltZSwgZGlnaXRzID0gbXlfZGlnaXRzKQ0KDQojID09PT09IEJMT0NLIDYgPT09PT0NCmNwcEZ1bmN0aW9uKCJkb3VibGUgZmFzdGVyNihOdW1lcmljVmVjdG9yIHByZWRzLCBOdW1lcmljVmVjdG9yIGxhYmVscywgZG91YmxlIGVwcykgew0KICBOdW1lcmljVmVjdG9yIGNsYW1wZWQgPSBjbGFtcChlcHMsIHByZWRzLCAxIC0gZXBzKTsNCiAgTnVtZXJpY1ZlY3RvciBsb2dneSA9IC0xICogKChsYWJlbHMgKiBsb2coY2xhbXBlZCkgKyAoMSAtIGxhYmVscykgKiBsb2coMSAtIGNsYW1wZWQpKSk7DQogIGRvdWJsZSBsb2dsb3NzID0gc3VtKGxvZ2d5KS9sb2dneS5zaXplKCk7DQogIHJldHVybiBsb2dsb3NzOw0KfSIpDQp0ZXN0X2Nhc2UoZmFzdGVyNiwgcHJlZHMgPSBkYXRhLCBsYWJlbHMgPSBsYWJlbHMsIGVwcyA9IDFlLTE1KQ0KZGF0YTYgPC0gcHJpbnRfd2VsbChtaWNyb2JlbmNobWFyayhmYXN0ZXI2KGRhdGEsIGxhYmVscywgZXBzID0gMWUtMTUpLCB0aW1lcyA9IG15X3J1bnMpJHRpbWUsIGRpZ2l0cyA9IG15X2RpZ2l0cykNCmBgYA0KDQojIFN1bW1hcnkgUmVzdWx0cw0KDQpgYGB7ciBiZW5jaDN9DQoNCmRhdGFfdGltZSA8LSBkYXRhLnRhYmxlKHJiaW5kbGlzdChsaXN0KGRhdGEuZnJhbWUoVGltZSA9IGRhdGExLCBCZW5jaCA9ICJmYXN0ZXIxIiksDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkYXRhLmZyYW1lKFRpbWUgPSBkYXRhMiwgQmVuY2ggPSAiZmFzdGVyMiIpLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGF0YS5mcmFtZShUaW1lID0gZGF0YTMsIEJlbmNoID0gImZhc3RlcjMiKSwNCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRhdGEuZnJhbWUoVGltZSA9IGRhdGE0LCBCZW5jaCA9ICJmYXN0ZXI0IiksDQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkYXRhLmZyYW1lKFRpbWUgPSBkYXRhNSwgQmVuY2ggPSAiZmFzdGVyNSIpLA0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGF0YS5mcmFtZShUaW1lID0gZGF0YTYsIEJlbmNoID0gImZhc3RlcjYiKSkpKQ0KZGF0YV90aW1lIDwtIGRhdGFfdGltZVssIHRfbWVhbiA6PSBtZWFuKFRpbWUpLCBieSA9IEJlbmNoXQ0KZGF0YV90aW1lIDwtIGRhdGFfdGltZVssIHRfbWVkaWFuIDo9IG1lZGlhbihUaW1lKSwgYnkgPSBCZW5jaF0NCmRhdGFfdGltZSRCZW5jaHMgPC0gZGF0YV90aW1lJEJlbmNoIA0KbGV2ZWxzKGRhdGFfdGltZSRCZW5jaHMpIDwtIHBhc3RlMCgiZmFzdGVyIiwgMTo2LCAiPSBbIiwgc3ByaW50ZihwYXN0ZTAoIiUuMCIsIG15X2RpZ2l0cywgImYiKSwgZGF0YV90aW1lWywgbGlzdChtaW4oVGltZSkpLCBieSA9IEJlbmNoXSRWMSksICIsICIsIHNwcmludGYocGFzdGUwKCIlLjAiLCBteV9kaWdpdHMsICJmIiksIGRhdGFfdGltZVssIGxpc3QobWF4KFRpbWUpKSwgYnkgPSBCZW5jaF0kVjEpLCAiXSwgbWVhbj0iLCBzcHJpbnRmKHBhc3RlMCgiJS4wIiwgbXlfZGlnaXRzLCAiZiIpLCBkYXRhX3RpbWVbLCBsaXN0KG1lYW4oVGltZSkpLCBieSA9IEJlbmNoXSRWMSksICIsIG1lZGlhbj0iLCBzcHJpbnRmKHBhc3RlMCgiJS4wIiwgbXlfZGlnaXRzLCAiZiIpLCBkYXRhX3RpbWVbLCBsaXN0KG1lZGlhbihUaW1lKSksIGJ5ID0gQmVuY2hdJFYxKSkNCg0KbXlfdGltZSA8LSBkYXRhX3RpbWVbLCBsaXN0KG1pbihUaW1lKSwgcXVhbnRpbGUoVGltZSwgcHJvYnMgPSAwLjI1KSwgbWVkaWFuKFRpbWUpLCBxdWFudGlsZShUaW1lLCBwcm9icyA9IDAuNzUpLCBtYXgoVGltZSksIG1lYW4oVGltZSkpLCBieSA9IEJlbmNoXQ0KY29sbmFtZXMobXlfdGltZSkgPC0gYygiRnVuY3Rpb24iLCAiTWluIiwgIjI1JSIsICI1MCUiLCAiNzUlIiwgIk1heCIsICJNZWFuIikNCm15X3RpbWUgPC0gbXlfdGltZVtvcmRlcihNZWFuLCBkZWNyZWFzaW5nID0gRkFMU0UpLCBdDQpwcmludChteV90aW1lLCBkaWdpdHMgPSA2KQ0KDQpgYGANCg0KIyBQbG90IFJlc3VsdHMNCg0KYGBge3IgYmVuY2g0LCBmaWcuaGVpZ2h0PTksIGZpZy53aWR0aD0xMH0NCg0KZ2dwbG90bHkoZ2dwbG90KGRhdGEgPSBkYXRhX3RpbWUsIGFlcyh4ID0gVGltZSkpICsgZ2VvbV9oaXN0b2dyYW0oYWVzKHkgPSAuLmRlbnNpdHkuLiksIGJpbnMgPSAyMCwgY29sb3IgPSAiZGFya2JsdWUiLCBmaWxsID0gImxpZ2h0Ymx1ZSIpICsgZmFjZXRfd3JhcCh+IEJlbmNocywgbmNvbCA9IDIpICsgZ2VvbV92bGluZShhZXMoeGludGVyY2VwdCA9IHRfbWVhbiksIGNvbG9yID0gImJsdWUiLCBsaW5ldHlwZSA9ICJkYXNoZWQiLCBzaXplID0gMikgKyBnZW9tX3ZsaW5lKGFlcyh4aW50ZXJjZXB0ID0gdF9tZWRpYW4pLCBjb2xvciA9ICJyZWQiLCBsaW5ldHlwZSA9ICJkYXNoZWQiLCBzaXplID0gMikgKyBsYWJzKHggPSAiVGltZSAoTWlsbGlzZWNvbmRzKSIsIHkgPSAiRGVuc2l0eSIpICsgdGhlbWVfYncoKSkNCg0KYGBgDQoNCmBgYHtyIGJlbmNoNSwgZmlnLmhlaWdodD02LCBmaWcud2lkdGg9MTB9DQpnZ3Bsb3RseShnZ3Bsb3QoZGF0YSA9IGRhdGFfdGltZVssIC4oVGltZSwgQmVuY2gpXSwgYWVzKHggPSBUaW1lLCB5ID0gLi5jb3VudC4uLCBmaWxsID0gQmVuY2gpKSArIGdlb21faGlzdG9ncmFtKGFlcyh5ID0gLi5kZW5zaXR5Li4pLCBiaW5zID0gMTAwLCBwb3NpdGlvbiA9ICJmaWxsIikgKyBsYWJzKHggPSAiVGltZSAoTWlsbGlzZWNvbmRzKSIsIHkgPSAiRGVuc2l0eSIpICsgdGhlbWVfYncoKSkNCmBgYA0KDQpgYGB7ciBiZW5jaDYsIGZpZy5oZWlnaHQ9NiwgZmlnLndpZHRoPTEwfQ0KZ2dwbG90bHkoZ2dwbG90KGRhdGEgPSBkYXRhX3RpbWVbLCAuKFRpbWUsIEJlbmNoKV0sIGFlcyh4ID0gVGltZSwgeSA9IC4uY291bnQuLiwgZmlsbCA9IEJlbmNoKSkgKyBnZW9tX2RlbnNpdHkocG9zaXRpb24gPSAiZmlsbCIpICsgbGFicyh4ID0gIlRpbWUgKE1pbGxpc2Vjb25kcykiLCB5ID0gIkRlbnNpdHkiKSArIHRoZW1lX2J3KCkpDQpgYGANCg0K